lib/commit: Include object type in sizes metadata

Append a byte encoding the OSTree object type for each object in the
metadata. This allows the commit metadata to be fetched and then for the
program to see which objects it already has for an accurate calculation
of which objects need to be downloaded.

This slightly breaks the `ostree.sizes` `ay` metadata entries. However,
it's unlikely anyone was asserting the length of the entries since the
array currently ends in 2 variable length integers. As far as I know,
the only users of the sizes metadata are the ostree test suite and
Endless' eos-updater[1]. The former is updated here and the latter
already expects this format.

1. https://github.com/endlessm/eos-updater/

diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c
index f88e2d78b2c5d3f2cc21e04bc984eaf4082e4b97..2294c84687b1f59a1dc999c05110a8172ba3b2b3 100644 (file)
--- a/src/libostree/ostree-repo-commit.c
+++ b/src/libostree/ostree-repo-commit.c
@@ -322,16 +322,19 @@ commit_loose_regfile_object (OstreeRepo        *self,
 /* This is used by OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES */
 typedef struct
 {
+  OstreeObjectType objtype;
   goffset unpacked;
   goffset archived;
 } OstreeContentSizeCacheEntry;
 
 static OstreeContentSizeCacheEntry *
-content_size_cache_entry_new (goffset unpacked,
-                              goffset archived)
+content_size_cache_entry_new (OstreeObjectType objtype,
+                              goffset          unpacked,
+                              goffset          archived)
 {
   OstreeContentSizeCacheEntry *entry = g_slice_new0 (OstreeContentSizeCacheEntry);
 
+  entry->objtype = objtype;
   entry->unpacked = unpacked;
   entry->archived = archived;
 
@@ -399,7 +402,7 @@ repo_store_size_entry (OstreeRepo       *self,
   repo_ensure_size_entries (self);
   g_hash_table_replace (self->object_sizes,
                         g_strdup (checksum),
-                        content_size_cache_entry_new (unpacked, archived));
+                        content_size_cache_entry_new (objtype, unpacked, archived));
 }
 
 static int
@@ -450,6 +453,7 @@ add_size_index_to_metadata (OstreeRepo        *self,
             g_hash_table_lookup (self->object_sizes, e_checksum);
           _ostree_write_varuint64 (buffer, e_size->archived);
           _ostree_write_varuint64 (buffer, e_size->unpacked);
+          g_string_append_c (buffer, (gchar) e_size->objtype);
 
           g_variant_builder_add (&index_builder, "@ay",
                                  ot_gvariant_new_bytearray ((guint8*)buffer->str, buffer->len));

diff --git a/tests/test-sizes.js b/tests/test-sizes.js
index 685fe3fe24c6f1784b72e4afdbeafc9103800e3e..a224653635111505c32129a4ea61b502b5b072dd 100755 (executable)
--- a/tests/test-sizes.js
+++ b/tests/test-sizes.js
@@ -67,15 +67,15 @@ function unpackByteArray(variant) {
     return array;
 }
 
-function validateSizes(repo, commit, expectedFiles) {
+function validateSizes(repo, commit, expectedObjects) {
     let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit);
     let metadata = commitVariant.get_child_value(0);
     let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay'));
-    let nSizes = sizes.n_children();
-    let expectedNSizes = Object.keys(expectedFiles).length
-    assertEquals(nSizes, expectedNSizes);
+    let nObjects = sizes.n_children();
+    let expectedNObjects = Object.keys(expectedObjects).length
+    assertEquals(nObjects, expectedNObjects);
 
-    for (let i = 0; i < nSizes; i++) {
+    for (let i = 0; i < nObjects; i++) {
         let sizeEntry = sizes.get_child_value(i);
         assertGreaterEquals(sizeEntry.n_children(), 34);
         let entryBytes = unpackByteArray(sizeEntry);
@@ -94,15 +94,20 @@ function validateSizes(repo, commit, expectedFiles) {
         assertGreaterEquals(remainingBytes.length, 1);
         [uncompressedSize, varintRead] = readVarint(remainingBytes);
         remainingBytes = remainingBytes.slice(varintRead);
-        assertEquals(remainingBytes.length, 0);
+        assertEquals(remainingBytes.length, 1);
+        let objectType = remainingBytes[0];
+        let objectTypeString = OSTree.object_type_to_string(objectType);
         print("compressed = " + compressedSize);
         print("uncompressed = " + uncompressedSize);
+        print("objtype = " + objectTypeString + " (" + objectType + ")");
+        let objectName = OSTree.object_to_string(checksumString, objectType);
+        print("object = " + objectName);
 
-        if (!(checksumString in expectedFiles)) {
-            throw new Error("Checksum " + checksumString + " not in " +
-                            JSON.stringify(expectedFiles));
+        if (!(objectName in expectedObjects)) {
+            throw new Error("Object " + objectName + " not in " +
+                            JSON.stringify(expectedObjects));
         }
-        let expectedSizes = expectedFiles[checksumString];
+        let expectedSizes = expectedObjects[objectName];
         let expectedCompressedSize = expectedSizes[0];
         let expectedUncompressedSize = expectedSizes[1];
         if (compressedSize != expectedCompressedSize) {
@@ -152,15 +157,26 @@ print("commit => " + commit);
 
 repo.commit_transaction(null);
 
-// Test the sizes metadata
-let expectedFiles = {
-    'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad': [54, 18],
-    'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b': [48, 12],
-    '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0': [43, 9],
-    '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51': [185, 185],
-    '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488': [12, 12],
+// Test the sizes metadata. The key is the object and the value is an
+// array of compressed size and uncompressed size.
+let expectedObjects = {
+    'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file': [
+        54, 18
+    ],
+    'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b.file': [
+        48, 12
+    ],
+    '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0.file': [
+        43, 9
+    ],
+    '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree': [
+        185, 185
+    ],
+    '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488.dirmeta': [
+        12, 12
+    ],
 };
-validateSizes(repo, commit, expectedFiles);
+validateSizes(repo, commit, expectedObjects);
 
 print("ok test-sizes");
 
@@ -168,9 +184,11 @@ print("ok test-sizes");
 // previous commit. Remove that file from the expected metadata and
 // replace the dirtree object.
 testDataDir.get_child('another-file').delete(null);
-delete expectedFiles['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad'];
-delete expectedFiles['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51'];
-expectedFiles['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539'] = [138, 138];
+delete expectedObjects['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file'];
+delete expectedObjects['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree'];
+expectedObjects['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539.dirtree'] = [
+    138, 138
+];
 
 repo.prepare_transaction(null);
 mtree = OSTree.MutableTree.new();
@@ -180,7 +198,7 @@ repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null);
 print("commit => " + commit);
 repo.commit_transaction(null);
 
-validateSizes(repo, commit, expectedFiles);
+validateSizes(repo, commit, expectedObjects);
 
 print("ok test-sizes file deleted");
 
@@ -194,6 +212,6 @@ repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null);
 print("commit => " + commit);
 repo.commit_transaction(null);
 
-validateSizes(repo, commit, expectedFiles);
+validateSizes(repo, commit, expectedObjects);
 
 print("ok test-sizes repeated");